from sklearn import base
from sklearn.dummy import DummyClassifier
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.feature_extraction import DictVectorizer
from sklearn.base import clone
from collections import defaultdict
import numpy
from scipy import sparse
from datetime import datetime
__author__ = 'panagiotis'
from sklearn.metrics.pairwise import cosine_distances as cosine
from collections import Counter

class ReadaptiveClustering(base.ClusterMixin):

    def __init__(self, threshold=1.0):
        self.clusters = numpy.array([])
        self.threshold = threshold

    def fit(self, vectors, threshold=None, debug=False):
        if sparse.issparse(vectors):
            vectors = vectors.todense()
        if threshold:
            self.threshold = threshold
        if debug:
            iteration = 0
        while True:
            clusters = []
            checked = set()
            i = 0
            while len(checked) < vectors.shape[0]:
                if i not in checked:
                    vec = vectors[i]
                    pos = numpy.where(pairwise_distances(vectors, vec, metric='cosine')[:, 0] <= (1-self.threshold))
                    move = [x for x in pos[0].tolist() if x not in checked]
                    if i not in move:
                        move.append(i)
                    checked |= set(move)
                    centroid = numpy.nan_to_num(vectors[move].sum(0)/numpy.sum(vectors[move] != 0, axis=0))
                    clusters.append(centroid)
                i += 1
                if debug:
                    print "process", round(100.0 * len(checked) / vectors.shape[0], 2)
            self.clusters = numpy.vstack(clusters)
            if vectors.shape == self.clusters.shape:
                return self
            else:
                if debug:
                    iteration += 1
                    print "iteration", iteration, ": number of clusters", self.clusters.shape[0]
                vectors = self.clusters

    def refit(self, vectors=None, threshold=None):
        if vectors:
            if hasattr(vectors, "shape"):
                v = vectors.todense()
            else:
                v = numpy.array(vectors)
            if v.shape[1] - self.clusters.shape[1] > 0:
                self.clusters = numpy.hstack((self.clusters, numpy.zeros((self.clusters.shape[0], v.shape[1] - self.clusters.shape[1]), dtype=self.clusters.dtype)))
            elif v.shape[1] - self.clusters.shape[1] < 0:
                raise AttributeError
            vectors = numpy.vstack((v, self.clusters))
        else:
            vectors = self.clusters
        if threshold:
            self.threshold = threshold
        while True:
            clusters = []
            checked = set()
            i = 0
            while len(checked) < vectors.shape[0]:
                if i not in checked:
                    vec = vectors[i]
                    pos = numpy.where(pairwise_distances(vectors, vec, metric='cosine')[:, 0] <= (1-self.threshold))
                    move = [x for x in pos[0].tolist() if x not in checked]
                    if i not in move:
                        move.append(i)
                    checked |= set(move)
                    centroid = numpy.nan_to_num(vectors[move].sum(0)/numpy.sum(vectors[move] != 0, axis=0))
                    clusters.append(centroid)
                i += 1
            self.clusters = numpy.vstack(clusters)
            if vectors.shape == self.clusters.shape:
                return self
            else:
                vectors = self.clusters

    def transform(self, vectors, debug=False):
        pos = pairwise_distances(self.clusters, vectors, metric='cosine')
        return self.clusters[numpy.argmin(pos, axis=0), :]

    def predict(self, vectors, debug=False):
        pos = pairwise_distances(self.clusters, vectors, metric='cosine')

        #temp = Counter()
        #for i in xrange(0, self.clusters.shape[0]):
        #    temp[i] = cosine(self.clusters[i], vectors)
        #return [temp.most_common()[-1][0]]
        return numpy.argmin(pos, axis=0).tolist()

    def fit_predict(self, vectors, threshold=None, debug=False):
        self.fit(vectors, threshold, debug=debug)
        return self.predict(vectors, debug=debug)

    def number_of_clusters(self):
        return self.clusters.shape[0]

    def recall(self, cid):
        return self.clusters[cid]


class NewArt():

    def __init__(self, classifier=None):
        self.user_vectorizer = None
        self.user_clusterer = ReadaptiveClustering()
        self.book_vectorizer = None
        self.book_clusterer = ReadaptiveClustering()
        if classifier:
            self.classifier_womb = classifier
        else:
            self.classifier_womb = DummyClassifier()
        self.classifiers = dict()

    def __transform(self, ratings, axis=0):
        if (axis not in [0, 1]) or (len(ratings[0]) != 3):
            raise ValueError
        temporary = defaultdict(dict)
        for score in ratings:
            temporary[score[axis]][score[1-axis]] = score[2]
        key_map = sorted(temporary.keys())
        vectorizer = DictVectorizer()
        vectors = vectorizer.fit_transform([temporary[user] for user in key_map])
        return key_map, vectors, vectorizer

    def fill_the_gaps(self, ratings):
        user_map, user_vectors, self.user_vectorizer = self.__transform(ratings, axis=0)
        user_map = zip(user_map, self.user_clusterer.fit_predict(user_vectors))
        user_map = {key: value for (key, value) in user_map}
        book_map, book_vectors, self.book_vectorizer = self.__transform(ratings, axis=1)
        book_map = zip(book_map, self.book_clusterer.fit(book_vectors))
        book_map = {key: value for (key, value) in book_map}

        ratings = [(user_map[user], book_map[book], score) for (user, book, score) in ratings]
        temporary = defaultdict(lambda: defaultdict(list))
        for (user, book, score) in ratings:
            temporary[user][book].append(score)
        for user in temporary.keys():
            X = []
            y = []
            for book in temporary[user].keys():
                X.append(self.book_clusterer.recall(book))
                y.append(int(round(sum(temporary[user][book]) / float(len(temporary[user][book])), 0)))
            try:
                clf = clone(self.classifier_womb)
                clf.fit(X, y)
            except ValueError:
                clf = DummyClassifier()
                clf.fit(X, y)
            finally:
                self.classifiers[user] = clf

        results = defaultdict(dict)
        for user in user_map.keys():
            X = []
            for book in sorted(book_map.keys()):
                X.append(self.book_clusterer.recall(book_map[book]))
            temp = self.classifiers[user_map[user]].predict(X).tolist()
            for (book, score) in [zip(sorted(book_map.keys()), temp)]:
                results[user][book] = score

        return results


class ArtMAP(base.ClassifierMixin):
    def __init__(self, classifier=None, choices=list):
        self.X_cluster = ReadaptiveClustering()
        self.fitted_X = False
        self.Y_cluster = ReadaptiveClustering()
        self.fitted_Y = False
        self.choices = choices
        if classifier:
            self.classifier_womb = classifier
        else:
            self.classifier_womb = DummyClassifier(strategy="stratified")
        self.Targets = []
        self.classifiers = dict()

    def _prefit_X(self, X, names=None, threshold=None):
    # Fit clustering algorithm of X beforehand
        self.X_cluster.fit(X, threshold=threshold)
        self.fitted_X = True

    def _prefit_Y(self, X, names=None, threshold=None):
    # Fit clustering algorithm of Y beforehand
        self.Y_cluster.fit(X, threshold=threshold)
        self.fitted_Y = True

    def fit(self, X, Y, z, threshold_x=None, threshold_y=None, debug=False):
        if sparse.isspmatrix(Y):
            Y = Y.tocsr()
        if self.choices == []:
            self.choices = sorted(set(z))
        if debug:
            t0 = datetime.now()
            print "starting ArtMAP fit process"
        if not self.fitted_X:
            self.X_cluster.fit(X, threshold=threshold_x)
        if not self.fitted_Y:
            self.Y_cluster.fit(Y, threshold=threshold_y)
        # for every target to be learned, predict the X cluster it belongs to
        # and append the centroid of the Y cluster it belongs to
        if debug:
            print "creating cluster to cluster MAP"

        ratings = defaultdict(lambda: defaultdict(list))
        s = 10000
        for i in xrange(0, len(Y), s):
            for (xcid, ycid, score) in zip(self.X_cluster.predict(X[i:i+s]), self.Y_cluster.predict(Y[i:i+s]), z[i:i+s]):
                ratings[xcid][ycid].append(score)
            if debug:
                    print "mapping iteration", i / s, "of", len(Y) / s
        #for i in xrange(0, len(Y)):
        #    xcid = self.X_cluster.predict(X[i])[0]
        #    ycid = self.Y_cluster.predict(Y[i])[0]
        #    score = z[i]
        #    ratings[xcid][ycid].append(score)
        print "mapped ratings"

        for xcid in ratings.keys():
            for ycid in ratings[xcid].keys():
                ratings[xcid][ycid] = sum(ratings[xcid][ycid]) / float(len(ratings[xcid][ycid]))
                if not isinstance(self.classifier_womb, base.RegressorMixin):
                    ratings[xcid][ycid] = int(round(ratings[xcid][ycid], 0))
        print "averaged values"
        # for every cluster in the X dimension(users), create a classifier of the given type
        if debug:
            print "fitting cluster classifiers"
        for xcid in ratings.keys():
            this_y = self.Y_cluster.recall(ratings[xcid].keys())
            this_z = [ratings[xcid][ycid] for ycid in ratings[xcid].keys()]
            try:
                # fit cluster's classifier with ratings gathered
                clf = clone(self.classifier_womb)
                clf.fit(this_y, this_z)
            except ValueError:
                # because some classifiers (if not all) demand that at least 2 classes exist,
                # a value error exception might appear. If this happens, a dummy classifier
                # replaces the predefined one, returning only one prediction
                print "ValueError exception in classifier fitting for cluster:", xcid
                clf = DummyClassifier(strategy="stratified")
                clf.fit(this_y, this_z)
            # trained classifier is stored
            self.classifiers[xcid] = clf
            if debug:
                print "cluster fitting progress", 100.0 * len(self.classifiers) / len(ratings)
        if debug:
            print "fitting completed in", str(datetime.now() - t0)

    def predict(self, X, Y, debug=False):
        # Predict the rating for every x,y
        z = []
        s = 1000
        if len(Y) > s:
            for i in xrange(0, len(Y), s):
                for (xcid, ycid) in zip(self.X_cluster.predict(X[i:i+s]), self.Y_cluster.transform(Y[i:i+s])):
                    try:
                        z.append(self.classifiers[xcid].predict(ycid))
                    except:
                        z.append(self.choices[0])
                    if debug:
                        print "model prediction progress", 100.0 * i / len(Y)
        else:
            nX = self.X_cluster.predict(X, debug=debug)
            nY = self.Y_cluster.transform(Y, debug=debug)
            for row in xrange(0, len(nX)):
                try:
                    z.append(self.classifiers[nX[row]].predict(nY[row])[0])
                except:
                    print "exception in classifier", nX[row]
                    z.append(self.choices[0])
                if debug:
                    print "model prediction progress", 100.0 * row / len(nX)
        return z

